from django.http import HttpResponse
import pymysql
import re
import sys
import math
import time
# from mythead import ThreadWithReturnValue

import random
import multiprocessing
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer

import numpy as np
import itertools
import numpy
import pandas
import json


def test():
    return 1


##
#总表（根据用户输入的期刊读取数据的表）
subjectTable = "wos_true"
#基金数量表
fundunitWeightTable = "fundunitweight"

weightArray = {
    "fundsFraction":7.5,
    "fundsNumFraction":5,
    "authorNumFraction":2.5,
    "authorAddrFraction":7.5,
    "authorCommuFraction":7.5,
    "referenceNumFraction":2.5,
    "referenceFraction":5,
    "pageFraction":2.5,
    "dtFraction":5,
    "laFraction":5,
    "j9Fraction":7.5,
    "zhyFraction":7.5,
    "altmetricsFraction":5,
    "z9Fraction":10,
    "downLoadsFraction":2.5,
    "yearMonth" : 1

}

#连接数据库
def mysqlConnect():
    # conn = pymysql.connect(host="localhost", user="root", passwd="", db="xkyc", charset="utf8")
    conn = pymysql.connect(host="localhost", user="root", passwd="kwssoft1234.", db="group_boundpu", charset="utf8")
    cur = conn.cursor()
    return conn,cur

#返回需要的字段
def subjectField():
    #出版年（py）、出版月（pd）、标题（ti）、期刊名称（j9）、出版类型（dt）、超链接（di）、语种（la）、页码（pg）、作者（au）、文献ID（PMID）、基金（FU）
    return "AB,DT,PY,PD,TI,J9,DT,DI,LA,PG,AU,FU,UT"

#返回基金权重表需要的字段

def fundunitWeightField():
    return "Weight"

#根据sql返回需要的数据
def sqlSelect(sql):

    conn, cur = mysqlConnect()
    try:
        cur.execute(sql)
    except:
        return []
    rows = cur.fetchall()
    cur.close()
    conn.close()

    return rows

#每一条结果得分格式
def resultFormS():

    data = {
        "ut":"",
        "fundsFraction":"",#基金质量分数   这里需要最后归一化！！！！！！！1
        "fundsNumFraction":"",# 基金数量分数
        "authorNumFraction": "",  # 作者数量分数
        "authorAddrFraction": "",  # 作者单位分数
        "authorCommuFraction":"", #通讯作者分数
        "referenceNumFraction": "",  # 文献数量
        "referenceFraction":"", #文献质量  这里需要最后归一化！！！！！！！1
        "pageFraction": "",  # 页码分数
        "dtFraction": "",  # 文献类型分数
        "laFraction": "",  # 文献类型分数
        "j9Fraction":"", #期刊影响因子分数   这里需要最后归一化
        "zhyFraction": "",  # 中科院分区
        "altmetricsFraction": "",  # 替代计量
        "z9Fraction": "",  #被引用次数   这里需要最后归一化
        "downLoadsFraction": "",  # 下载次数   这里需要最后归一化
        "yearMonth": "",  # 年份得分
        "year":"",
        "month":""
    }

    return data
##########################读取对应表的数据########################################################################################################

# 根据用户输入的期刊得到数据
def getSubjectData( J9=''):

    sql = "select {} from {} where J9 in({})".format(subjectField(), subjectTable,J9)

    # sql = "select {} from {} limit 2".format(subjectField(), subjectTable, J9)

    data = sqlSelect(sql)

    return data

# 得到基金权重的数据
def getFundWeightData( str=''):

    sql = '''select {} from {} where `Fund` in ({})'''.format(fundunitWeightField(), fundunitWeightTable,str)

    data = sqlSelect(sql)

    return data

#拆分
def spintFund(str):

    str = str.replace(';-;',' ')
    temp1 = str.split('; ')
    dd = clearFund(temp1)
    return dd


# 清除基金的[]
def clearFund( data):
    temp = []
    for dd in data:
        temp.append(re.sub(r'\[.*\]', '', dd).strip())
    return temp

#pmid转json
def pmidToJson(data):
    jsonData = {}
    for dd in data:
        jsonData[dd["ut"]] = dd["allScore"]

    return jsonData

# def formatTiDif(data):
#
#     allJsonData = {}
#     pmidArray = []
#     for dd in data:
#         pmidArray.append(dd[0])
#
#     pmidArray = (list(set(pmidArray)))
#
#     for pmid in pmidArray:
#         tempJson = {}
#         for dd in data:
#             if pmid == dd[0]:
#
#                 tempJson[dd[1]] = dd[2]
#             else:
#                 continue
#         allJsonData[pmid] = tempJson
#
#     return allJsonData

#格式化tfidf字典
def formatTiDif(data):
    allJsonData = {}
    pmidArray = []
    for dd in data:
        pmidArray.append(dd[0])

    pmidArray = (list(set(pmidArray)))

    data = (pandas.DataFrame(data, columns=["pmid", "keyword", "score"]))

    # --------------------------------------------------------------------------------------------------------

    #
    cpuNum = 32
    pool = multiprocessing.Pool(processes=cpuNum)

    offset = int(len(pmidArray) / cpuNum)

    results = []
    for i in range(cpuNum):
        if i == 0:
            start = 0

        if i == cpuNum - 1:
            end = 0

        end = start + offset

        res = pool.apply_async(formatTiDifProcess,
                               (pmidArray[start:end], data))  # 异步开启进程, 非阻塞型, 能够向池中添加进程而不等待其执行完毕就能再次执行循环
        results.append(res)

        start += offset

    pool.close()  # 关闭pool, 则不会有新的进程添加进去
    pool.join()  # 必须在join之前close, 然后join等待pool中所有的线程执行完毕

    # print(results)
    dd3 = {}
    for i in results:
        dd = i.get()  # 获得进程的执行结果
        try:
            dd3.update(dd)
        except:
            # print(dd)
            continue

    # for pmid in pmidArray:
    #     tempJson = {}
    #     tempData = numpy.array(data[data["pmid"] == str(pmid)])
    #     for dd in tempData:
    #         tempJson[dd[1]] = dd[2]
    #
    #     allJsonData[pmid] = tempJson

    # for pmid in pmidArray:
    #     tempJson = {}
    #     for dd in data:
    #         if pmid == dd[0]:
    #
    #             tempJson[dd[1]] = dd[2]
    #         else:
    #             continue
    #     allJsonData[pmid] = tempJson


    return dd3

def formatTiDifProcess(pmidArray, data):

    allJsonData = {}
    for pmid in pmidArray:
        tempJson = {}
        tempData = numpy.array(data[data["pmid"] == str(pmid)])
        for dd in tempData:
            tempJson[dd[1]] = dd[2]

        allJsonData[pmid] = tempJson

    return allJsonData

#给关键字赋予分数转成字典格式 改
def keyWordToJson(keyWordData,pmidJson,tfidfArray):
    # 优化用的allDataArray
    allDataArray = []

    # print("qwer1")
    start = time.time()
    tfidfArray = formatTiDif(tfidfArray)
    end = time.time()
    # print(end - start)

    # print("qwer2")
    #用来存关键字和分数构成的字典
    allData = []

    #用来存关键字
    allKeyword = []

    # print(keyWordData)
    # print(pmidJson)
    # exit()
    # print(len(keyWordData))
    # return keyWordData

    for dd in keyWordData:


        #!!!!!这里同一个关键字只会出现在标题、摘要或者关键词中
        #第一个是标题中出现
        # keyArray = list(dd)[1].split("|")

        # allKeyword.extend(keyArray)

        temp = {}
        # for key in keyArray:
        #     try:
        #         temp[key] = 1 * pmidJson[dd[0]] + tfidfArray[dd[0]][key]
        #     except:
        #         temp[key] = 1 * pmidJson[dd[0]]



        # 第三个是关键词中出现
        keyArray = list(dd)[3].split("|")
        allKeyword.extend(keyArray)

        for key in keyArray:
            try:
                temp[key] = 0.9 * pmidJson[dd[0]] + tfidfArray[dd[0]][key]
            except:
                temp[key] = 1 * pmidJson[dd[0]]

        # 第二个是摘要中出现
        # keyArray = list(dd)[2].split("|")
        # allKeyword.extend(keyArray)
        #
        # for key in keyArray:
        #     try:
        #         temp[key] = 0.8 * pmidJson[dd[0]] + tfidfArray[dd[0]][key]
        #     except:
        #         temp[key] = 1 * pmidJson[dd[0]]

        for key, value in temp.items():
            allDataArray.append([key.lower(), value])

        allData.append(temp)

    return allDataArray


#构造关键词的字典
def formKeyJson(data):
    json = {}
    for dd in data:
        json[dd] = 0

    return json

#构建用户输入的期刊和权重的字典
def formJson(j9, weight):
    jsonData = {}

    j9Array = j9.split(",")
    weightArray = weight.split(",")

    for i in range(len(j9Array)):
        jsonData[j9Array[i]] = weightArray[i]

    return jsonData

#构建查询期刊需要的字符串
def j9JsonData(j9WeightJson):
    str = ""
    for key in j9WeightJson.keys():
        tempStr = "'" + key + "'"
        str += tempStr + ","

    return str.strip(",")

#
def createData(data):
    allData = []
    for dd in data:
        if len(dd[0]) < 5:
            continue
        tempData = {
            "keyword": "",
            # "score": ""
        }

        tempData["keyword"] = dd[0]
        # tempData["score"] = dd[1]
        allData.append(tempData)

    try:
        return allData[0:500]
    except:
        return allData

def getKeyExtend(data):
    pmidStr = ''

    allStr = []
    i = 0

    for dd in data:

        i += 1
        str = "'" + dd["ut"] + "'"
        pmidStr += str + ","
        if i == 500:
            i = 0
            allStr.append(pmidStr.strip(","))
            pmidStr = ''

    allStr.append(pmidStr.strip(","))

    # pmidStr = pmidStr.strip(",")
    allData = []
    for str in allStr:
        sql = "select * from extractkey where ut in ({}) and length(keyKey)>5".format(str)

        allData.extend(sqlSelect(sql))

    return allData

#计算tfidf
def getTfIdf(data):
    # print(data)
    pmidArray = []
    keyWordArray = []

    for dd in data:
        pmidArray.append(dd[0])
        # 构造字符串
        # print(dd)
        replaceStr = ["-", "[", "]", "(", ")"]
        # 词组之间用_隔开 不同词或者词组用空格隔开
        str = (dd[3]).replace(" ", "_").replace(';_', ' ')
        # str = str.replace("-","_").replace("[","_").replace("]","_").replace("(","_").replace(")","_")
        # print(str)
        # exit()
        keyWordArray.append(str)



    vectorizer = CountVectorizer()  # 该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
    transformer = TfidfTransformer()  # 该类会统计每个词语的tf-idf权值
    tfidfArray = transformer.fit_transform(
        vectorizer.fit_transform(keyWordArray))  # 第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵
    word = vectorizer.get_feature_names()  # 获取词袋模型中的所有词语

    # print(tfidfArray.toarray())
    # exit()

    allData = []
    i = 0
    for tfidf in tfidfArray:
        if i % 10000 == 0:
            pass
        weight = tfidf.toarray()  # 将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重
        # 获取不等于0的下标
        notZero = (np.where(weight > 0))

        for notZeroIndex in notZero[1]:
            temp = []
            temp.append(pmidArray[i])
            temp.append(word[notZeroIndex].replace("_", " "))
            temp.append(weight[0][notZeroIndex])
            allData.append(temp)

        i += 1

    return allData

def getTfIdfZ(data):

    keyWordArray = []

    for dd in data:
        tempData = []
        tempData.append(dd[0])
        tempData.append(formKeyWorsToTf(dd[1:]))
        keyWordArray.append(tempData)


    #判断这个关键字是否已经计算过

    allData = []

    for keyWords in keyWordArray:
        isExitArray = []
        for keyWord in keyWords[1]:
            if keyWord in isExitArray:
                continue
            isExitArray.append(keyWord)
            tempTfIdf = []
            tempTfIdf.append(keyWords[0])
            tempTfIdf.append(keyWord)
            tempTfIdf.append(calTf(keyWord,keyWords[1]) * calIdf(keyWord, keyWordArray))

            allData.append(tempTfIdf)
    return allData


#计算tf
def calTf(keyWord,keyWords):
    num = 0
    for dd in keyWords:
        if keyWord == dd:
            num += 1
    return num / len(keyWords)

#计算idf
def calIdf(keyWord,keyWordArray):
    num = 0
    for keyWords in keyWordArray:
        if keyWord in keyWords[1]:
            num +=1

    return math.log10(len(keyWordArray)/num)

#处理关键词
def formKeyWorsToTf(keyWord):
    keyWordStr = list("|".join(keyWord).split("|"))
    return keyWordStr

def calMore(subjectData,j9WeightJson):

    allData = []
    allStrArray = []

    for subject in subjectData:
        try:
            resultForm = resultFormS()


            resultForm['ut'] = subject[-1]
            resultForm['fundsFraction'] = calFunds(subject[-2])

            resultForm['fundsNumFraction'] = calFundsNum(subject[-2])
            resultForm['authorNumFraction'] = calAuthorNum(subject[-3])

            resultForm['authorAddrFraction'] = calAuthorAddrNum(subject[-1])  # 作者单位

            resultForm['authorCommuFraction'] = calAuthorCommuFraction(subject[-1])  # 作者通讯

            resultForm['referenceNumFraction'], resultForm['referenceFraction'] = caleFerenceFraction(
                subject[-1])  # 文献数量和质量一起返回

            resultForm['pageFraction'] = calPageFraction(subject[-4])  # 页码
            resultForm['dtFraction'] = calDtFraction(subject[-12])  # 文献类型
            resultForm['laFraction'] = calLaFraction(subject[-5])  # 语种类型

            resultForm['j9Fraction'] = calJ9Fraction(subject[-8], j9WeightJson)  # 期刊影响因子
            resultForm['zhyFraction'] = calZhyFraction(subject[-8])  # 中科院分区
            resultForm['altmetricsFraction'] = calAltFraction(subject[-1])  # 替代计量指数

            resultForm['downLoadsFraction'] = calDownFraction(subject[-1])  # 下载次数
            resultForm['z9Fraction'] = calZ9Fraction(subject[-1])  # 被引用次数

            # 存储年月 方便后面计算
            resultForm['year'] = subject[-11]
            resultForm['month'] = subject[-10]

            resultForm['yearMonth'] = calYearMontnScore(resultForm)

            allData.append(resultForm)
            allStrArray.extend([subject[-9] + " " + subject[-13]])
        except:
            pass

    return allData,allStrArray

def calKeyWordV2(allKeyword,allData):
    allDataN = {}

    for dd in allKeyword:
        allDataN[dd] = 0

    for key in allKeyword:
        for dd in allData:
            try:
                allDataN[key] += dd[key]

                # print(dd[key])

            except:
                allDataN[key] += 0

    return allDataN

#sort Json
def sortjson(dataJson,newKeys):

    newJson = {}
    result = sorted(dataJson.items(), key=lambda item: item[1])[-200:]

    for dd in result:
        newJson[dd[0]] = dd[1]

    for key in newKeys:
        newJson[key[0]] = dataJson[key[0]]

    return newJson

###########################主程序#######################################################################################################

def index(request):

    # return HttpResponse([1], content_type="application/json")
    #用户输入的 改 还有权重（局部
    # J9 = "JAMA PEDIATR,J AM ACAD CHILD PSY,PEDIATRICS,PEDIATR OBES,ARCH DIS CHILD-FETAL,EUR CHILD ADOLES PSY,SEMIN FETAL NEONAT M,DEV MED CHILD NEUROL,ARCH DIS CHILD,PEDIATR PULM,PEDIATR RES,J CHILD ADOL PSYCHOP,PEDIATR CLIN N AM,ACAD PEDIATR,J PEDIATR GASTR NUTR,CURR OPIN PEDIATR,PEDIATR BLOOD CANCER,PEDIATR NEPHROL,ACTA PAEDIATR,CHILD OBES,PAEDIATR PERINAT EP,DEV PSYCHOBIOL,J PEDIAT INF DIS SOC,PEDIATR NEUROL,PAEDIATR RESPIR REV,PEDIATR INFECT DIS J,EUR J PEDIATR,J DEV BEHAV PEDIATR,J ADOLESC YOUNG ADUL,J NEUROSURG-PEDIATR,J PEDIATR SURG,BMC PEDIATR,EARLY HUM DEV,PEDIATR DRUGS,J PEDIATR UROL,SEMIN PEDIATR NEUROL,PEDIATR RADIOL,J PEDIATR NURS,ITAL J PEDIATR,J PEDIATR ADOL GYNEC,PHYS OCCUP THER PEDI,CLIN CHILD PSYCHOL P,CHILD CARE HLTH DEV,J PEDIAT-BRAZIL,J CHILD NEUROL,SEMIN PEDIATR SURG,NEUROPEDIATRICS,ARCH DIS CHILDHOOD-E,BRAIN DEV-JPN,PEDIATR CARDIOL,PAEDIATR INT CHILD H,EUR J PEDIATR SURG,CURR PROB PEDIATR AD,PEDIATR SURG INT,J PAEDIATR CHILD H,CLIN PEDIATR,APPL NEUROPSYCH-CHIL,INT J PAEDIATR DENT,PEDIATR TRANSPLANT,PEDIATR EXERC SCI,DEV NEUROREHABIL,INT J PEDIATR OTORHI,J PEDIATR ONCOL NURS,PEDIATR NEONATOL,WORLD J PEDIATR,J TROP PEDIATRICS,J CLIN RES PEDIATR E,PEDIATR HEMAT ONCOL,INDIAN PEDIATR,J CHILD HEALTH CARE,PEDIATR ENDOCR REV P,PEDIATR EMERG CARE,J PEDIAT HEMATOL ONC,INDIAN J PEDIATR,PEDIATR DERMATOL,CARDIOL YOUNG,MINERVA PEDIATR,J AAPOS,PEDIATR PHYS THER,EUR J PAEDIATR DENT,J SPEC PEDIATR NURS,PEDIATR INT,J CLIN PEDIATR DENT,PEDIATR NEUROSURG,KLIN PADIATR,PEDIATR ANN,J PEDIATR ORTHOP B,FETAL PEDIATR PATHOL,ARCH ARGENT PEDIATR"
    #
    # #用户输入的权重
    # weight = "1,0.9,1,0.9,0.9,0.9,0.9,0.9,0.9,0.9,1,0.8,0.8,1,0.8,0.8,0.8,0.8,1,0.8,0.8,0.8,0.8,0.8,0.7,0.7,1,0.7,0.7,0.7,0.7,0.7,0.5,0.7,0.7,0.7,0.7,0.6,0.6,0.6,0.6,0.6,0.6,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.1,0.1"
    # 用户输入的 改 还有权重（局部
    J9 = request.POST.get('J9', 'abc,f')

    # 用户输入的权重
    weight = request.POST.get('weight', '1,4')

    # J9 = "JAMA PEDIATR,J AM ACAD CHILD PSY,PEDIATRICS,PEDIATR OBES,ARCH DIS CHILD-FETAL,EUR CHILD ADOLES PSY,SEMIN FETAL NEONAT M,DEV MED CHILD NEUROL,ARCH DIS CHILD,PEDIATR PULM,PEDIATR RES,J CHILD ADOL PSYCHOP,PEDIATR CLIN N AM,ACAD PEDIATR,J PEDIATR GASTR NUTR,CURR OPIN PEDIATR,PEDIATR BLOOD CANCER,PEDIATR NEPHROL,ACTA PAEDIATR,CHILD OBES,PAEDIATR PERINAT EP,DEV PSYCHOBIOL,J PEDIAT INF DIS SOC,PEDIATR NEUROL,PAEDIATR RESPIR REV,PEDIATR INFECT DIS J,EUR J PEDIATR,J DEV BEHAV PEDIATR,J ADOLESC YOUNG ADUL,J NEUROSURG-PEDIATR,J PEDIATR SURG,BMC PEDIATR,EARLY HUM DEV,PEDIATR DRUGS,J PEDIATR UROL,SEMIN PEDIATR NEUROL,PEDIATR RADIOL,J PEDIATR NURS,ITAL J PEDIATR,J PEDIATR ADOL GYNEC,PHYS OCCUP THER PEDI,CLIN CHILD PSYCHOL P,CHILD CARE HLTH DEV,J PEDIAT-BRAZIL,J CHILD NEUROL,SEMIN PEDIATR SURG,NEUROPEDIATRICS,ARCH DIS CHILDHOOD-E,BRAIN DEV-JPN,PEDIATR CARDIOL,PAEDIATR INT CHILD H,EUR J PEDIATR SURG,CURR PROB PEDIATR AD,PEDIATR SURG INT,J PAEDIATR CHILD H,CLIN PEDIATR,APPL NEUROPSYCH-CHIL,INT J PAEDIATR DENT,PEDIATR TRANSPLANT,PEDIATR EXERC SCI,DEV NEUROREHABIL,INT J PEDIATR OTORHI,J PEDIATR ONCOL NURS,PEDIATR NEONATOL,WORLD J PEDIATR,J TROP PEDIATRICS,J CLIN RES PEDIATR E,PEDIATR HEMAT ONCOL,INDIAN PEDIATR,J CHILD HEALTH CARE,PEDIATR ENDOCR REV P,PEDIATR EMERG CARE,J PEDIAT HEMATOL ONC,INDIAN J PEDIATR,PEDIATR DERMATOL,CARDIOL YOUNG,MINERVA PEDIATR,J AAPOS,PEDIATR PHYS THER,EUR J PAEDIATR DENT,J SPEC PEDIATR NURS,PEDIATR INT,J CLIN PEDIATR DENT,PEDIATR NEUROSURG,KLIN PADIATR,PEDIATR ANN,J PEDIATR ORTHOP B,FETAL PEDIATR PATHOL,ARCH ARGENT PEDIATR"
    #
    # #用户输入的权重
    # weight = "1,0.9,1,0.9,0.9,0.9,0.9,0.9,0.9,0.9,1,0.8,0.8,1,0.8,0.8,0.8,0.8,1,0.8,0.8,0.8,0.8,0.8,0.7,0.7,1,0.7,0.7,0.7,0.7,0.7,0.5,0.7,0.7,0.7,0.7,0.6,0.6,0.6,0.6,0.6,0.6,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.1,0.1"

    #这里构建一个期刊权重字典
    J9 = J9.strip(",")
    weight = weight.strip(",")

    j9WeightJson = formJson(J9,weight)

    #新的J9字符串
    J9 = j9JsonData(j9WeightJson)

    #获得需要的学科数据
    subjectData = getSubjectData(J9)


    subjectData = list(subjectData)
    random.shuffle(subjectData)

    subjectData = subjectData
    # print(len(subjectData))
    #allStrArray为了在做共词的时候，先把标题和摘要存进去
    allStrArray = []

    allData = []

#--------------------------------------------------------------------------------------------------------
    # middel = int(len(subjectData) / 2)
    #
    cpuNum = 16
    pool = multiprocessing.Pool(processes=cpuNum)

    offset = int(len(subjectData) / cpuNum)

    results = []
    for i in range(cpuNum):
        if i == 0:
            start = 0

        if i == cpuNum - 1:
            end = 0

        end = start + offset

        res = pool.apply_async(calMore,(subjectData[start:end], j9WeightJson))  # 异步开启进程, 非阻塞型, 能够向池中添加进程而不等待其执行完毕就能再次执行循环
        results.append(res)

        start += offset


    pool.close()  # 关闭pool, 则不会有新的进程添加进去
    pool.join()  # 必须在join之前close, 然后join等待pool中所有的线程执行完毕

    # print(results)

    for i in results:
        oneData, twoData = i.get()  # 获得进程的执行结果
        try:
            allData.extend(oneData)
            allStrArray.extend(twoData)
        except:
            raise
            pass


# --------------------------------------------------------------------------------------------------------
    # exit()
    allStrArray = (list(set(allStrArray)))
    # print("20")
    #归一化
    data = normalization(allData)

    # print("10")
    #乘权重
    data = calWeight(data)
    # print("30")
    #计算总分
    data = calTotalScore(data)

    # print("40")

    #计算年月
    # data = calYearMontnScore(data)

    #计算关键字分数keyWordData为关键字数据 用于计算tfidf


    keyWordData = getKeyExtend(data)
    # return HttpResponse([keyWordData], content_type="application/json")

    if (len(keyWordData) == 0):
        return HttpResponse(json.dumps({
            "collection": []
        }
        ), content_type="application/json")

    tfidfArray = getTfIdf(keyWordData)

    # print("start calKeyWordCal")
    dataKeyWord = calKeyWordCal(data,tfidfArray,keyWordData)



    #计算每个关键词的tfidf值

    # #两两共词,改
    # return HttpResponse([dataKeyWord], content_type="application/json")
    newKeys = getNewKey(dataKeyWord)

    data = coWordTwo(dataKeyWord,newKeys)

    # return HttpResponse([data], content_type="application/json")
    #三三共词
    # data = coWordThree(allStrArray,data)

    # print(data)
    # exit()

    data = getResult(newKeys, data)

    # print("getResult")
    # return HttpResponse([data], content_type="application/json")

    result = sorted(data.items(),key=lambda item:item[1])[::-1]


    data = createData(result)

    data = [data[i] for i in range(len(data)) if i % 2 == 0][0:500]
    # return HttpResponse([1], content_type="application/json")

    return HttpResponse(json.dumps({
        "collection":data,
        "newKeyWords": [dd[0] for dd in newKeys]
    }
    ), content_type="application/json")
    # print("ffff")
    # print(data)

def getResultProcess(dataKeys,data,newKeys):
    newJson = {}
    for dd in dataKeys:
        splitA = dd.split("|")
        for k in splitA:
            if k in newKeys:
                newJson[dd] = data[dd]

    return newJson

def structureNewKeys(newKeys):
    newArray = []
    for dd in newKeys:
        newArray.append(dd[0])

    return newArray

def getResult(newKeys,data):

    newKeys = structureNewKeys(newKeys)
    # data = getResultProcess(newKeys[0:900],data)
    # print(data)
    # print("5455x")
    # exit()
    #    -----------------------------------------------------------
    dataKeys = list(data.keys())

    cpuNum = 16
    pool = multiprocessing.Pool(processes=cpuNum)

    offset = int(len(dataKeys) / cpuNum)

    results = []
    for i in range(cpuNum):
        if i == 0:
            start = 0

        if i == cpuNum - 1:
            end = 0

        end = start + offset

        res = pool.apply_async(getResultProcess, (dataKeys[start:end],data,newKeys))  # 异步开启进程, 非阻塞型, 能够向池中添加进程而不等待其执行完毕就能再次执行循环
        results.append(res)

        start += offset

    pool.close()  # 关闭pool, 则不会有新的进程添加进去
    pool.join()  # 必须在join之前close, 然后join等待pool中所有的线程执行完毕

    # print(results)

    dd3 = {}
    for i in results:
        dd = i.get()  # 获得进程的执行结果
        try:
            dd3.update(dd)
        except:
            # print(dd)
            continue


            # -----------------------------------------------------------

    return dd3


def getNewKeyProcess(currentData):



    limit = 1000
    start = 0
    allData = []

    while 1:

        data = currentData[start:start + limit]
        if len(data) == 0:
            break

        strKey = ""
        for dd in data:
            tempD = pymysql.escape_string(dd)
            tempD = '"' + tempD + '"'
            strKey += tempD + ","

        strKey = (strKey.strip(","))

        sql = "select keyWord from allkeyword where isNew=1 and keyWord in (%s)" % strKey

        try:
            data = list(sqlSelect(sql))
        except:
            raise
            pass
        allData.extend(data)

        start += limit


    return allData


def getNewKey(data):

    allKey = data.keys()
    subjectData = list(allKey)
    # allData = getNewKeyProcess(subjectData[0:10000])
    # # print(allData)
    # exit()
    allData = []
#    -----------------------------------------------------------
    cpuNum = 16
    pool = multiprocessing.Pool(processes=cpuNum)

    offset = int(len(subjectData) / cpuNum)

    results = []
    for i in range(cpuNum):
        if i == 0:
            start = 0

        if i == cpuNum - 1:
            end = 0

        end = start + offset

        res = pool.apply_async(getNewKeyProcess,([subjectData[start:end]]))  # 异步开启进程, 非阻塞型, 能够向池中添加进程而不等待其执行完毕就能再次执行循环
        results.append(res)

        start += offset

    pool.close()  # 关闭pool, 则不会有新的进程添加进去
    pool.join()  # 必须在join之前close, 然后join等待pool中所有的线程执行完毕

    # print(results)

    for i in results:
        oneData = i.get()  # 获得进程的执行结果

        try:
            allData.extend(oneData)
        except:
            raise
            pass


        # -----------------------------------------------------------
    return allData

# 构造三三共词需要的字典形式
def createCoWordThreeJson( data):

    keyJson = {}
    for dd in data:
        for cc in data:
            for ee in data:
                keyJson[dd + " " + cc + " " + ee] = dd + "|" + cc + "|" + ee

    return keyJson

#构造两两共词需要的字典形式
def createCoWordTwoJson(data):
    keyJson = {}
    for dd in data:
        for cc in data:
            keyJson[dd + " " + cc] = dd + "|" + cc

    return keyJson

#利用笛卡尔积组合词
def combKeyWord(data):

    c = itertools.product(data, data)

    # keyJson = {}
    #
    # for i in c:
    #     key = " ".join(i)
    #     keyJson[key] = "|".join(i)

    return list(c)

#两两分词多进程
def coWordTwoProcess(keyWordsArray, data):
    newJson = {}
    for dd in keyWordsArray:
        if dd[0] == dd[1]:
            continue
        else:
            try:
                num = data[dd[0]] + data[dd[1]]
            except:
                num = 0
            newJson[dd[0] + "|" + dd[1]] = num

    return newJson
#两两共词
def coWordTwo(data,newKeys):

    newData = sortjson(data,newKeys)
    # return newData
    # 提取出所有关键字并且构造两两共词需要的字典形式
    keyWords = list(newData.keys())

    # return keyWords

    keyWordsArray = combKeyWord(keyWords)

# --------------------------------------------------------------------------------------------------------

    #
    cpuNum = 16
    pool = multiprocessing.Pool(processes=cpuNum)

    offset = int(len(keyWordsArray) / cpuNum)

    results = []
    for i in range(cpuNum):
        if i == 0:
            start = 0

        if i == cpuNum - 1:
            end = 0

        end = start + offset

        res = pool.apply_async(coWordTwoProcess,
                               (keyWordsArray[start:end], data))  # 异步开启进程, 非阻塞型, 能够向池中添加进程而不等待其执行完毕就能再次执行循环
        results.append(res)

        start += offset

    pool.close()  # 关闭pool, 则不会有新的进程添加进去
    pool.join()  # 必须在join之前close, 然后join等待pool中所有的线程执行完毕

    # print(results)
    dd3 = {}
    for i in results:
        dd = i.get()  # 获得进程的执行结果
        try:
            dd3.update(dd)
        except:
            # print(dd)
            continue

    return dd3


# 三三共词
def coWordThree(allStrArray,data):

    # data = {"hello": 1, "world": 1, "world1": 1, "no": 2,"faf":9}

    # 提取出所有关键字并且构造两两共词需要的字典形式
    keyWords = list(data.keys())
    keyWordsJson = createCoWordThreeJson(keyWords)

    # test
    # keyWordsJson = {
    #     "hello world faf": "hello|world|faf",
    #     "no world faf": "no|world|faf",
    # }
    #
    # allStrArray = [
    #     "hello world faf afaef fae",
    #     "hello world1 faf afaef fae",
    #     "hello wold faf afaef fae",
    #     "no world faf afaef fae",
    #
    # ]

    # 记录比如：（{'fsdfs af': 1, 'effe hello': 1}）
    numKeyJson = {}

    for key in keyWordsJson.keys():
        for str in allStrArray:
            if key in str:
                try:
                    numKeyJson[key] += 1
                except:
                    numKeyJson[key] = 2

    # 如果numKeyJson == 0 说明没有关键词一起出现过
    if (len(numKeyJson)) == 0:
        return data

    newData = {}
    # 根据关键词计算分数
    for key, value in data.items():
        for key1, value1 in numKeyJson.items():
            newData[key1] = 0

            if key in key1:
                data[key] *= value1

    # 计算组合词分数
    for key, value in data.items():
        for key1, value1 in newData.items():
            if key in key1:
                try:
                    newData[key1] += data[key]
                except:
                    newData[key1] = 0

    # 合并字典
    for key1, value1 in newData.items():
        data[key1] = value1

    print(data)
    exit()

    return data

    print(keyWordsJson)
    exit()

#计算关键字分数
def calKeyWordCal(data,tfidfArray,keyWordData):

    # print(keyWordData)
    #构造pmid为键 总分为值的字典方便后面赋予关键字分数
    pmidJson = pmidToJson(data)

    # 给关键字赋予分数并返回字典格式和所有关键字构成的数组
    allDataArray = keyWordToJson(keyWordData, pmidJson, tfidfArray)
    # return allDataArray
    # return HttpResponse([1], content_type="application/json")

    data = (pandas.DataFrame(allDataArray, columns=["key", "score"]))

    data = (data.groupby("key").sum())

    dict_country = data.T.to_dict('list')

    for key, value in dict_country.items():
        dict_country[key] = value[0]

    return dict_country
    # 构造关键词的字典
    dataJson = formKeyJson(allKeyword)

# -----------------------------------------------------------
    cpuNum = 16
    pool = multiprocessing.Pool(processes=cpuNum)
    offset = int(len(allKeyword) / cpuNum)

    results = []
    for i in range(cpuNum):
        if i == 0:
            start = 0

        if i == cpuNum - 1:
            end = 0

        end = start + offset

        res1 = pool.apply_async(calKeyWordV2,
                                (allKeyword[start:end], allData))  # 异步开启进程, 非阻塞型, 能够向池中添加进程而不等待其执行完毕就能再次执行循环
        results.append(res1)

        start += offset

    pool.close()  # 关闭pool, 则不会有新的进程添加进去
    pool.join()  # 必须在join之前close, 然后join等待pool中所有的线程执行完毕

    # print(results)

    dd3 = {}

    for i in results:

        dd = i.get()  # 获得进程的执行结果
        # print(dd)
        try:
            dd3.update(dd)
        except:
            # print(dd)
            raise
            pass

    return dd3
#------------------------------------------------------------------------
    # for key in allKeyword:
    #     for dd in allData:
    #         try:
    #             dataJson[key] += dd[key]
    #         except:
    #             pass

    # return dataJson



#计算年月分数 改
def calYearMontnScore(dd):

    yearWeigthArray = {
        # "2009": [0.6875, 0.715, 0.7425, 0.77, 0.7975, 0.825, 0.8525, 0.88, 0.9075, 0.935, 0.9625, 1],
        "2016": [0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.175, 0.2, 0.225, 0.25, 0.275, 0.3],
        "2017": [0.3445, 0.371, 0.3975, 0.424, 0.4505, 0.477, 0.5035, 0.53, 0.5565, 0.583, 0.6095, 0.636],
        "2018": [0.6875, 0.715, 0.7425, 0.77, 0.7975, 0.825, 0.8525, 0.88, 0.9075, 0.935, 0.9625, 1]
    }

    monthEnglish = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]



    if dd["year"] in ",".join(yearWeigthArray.keys()):
        if dd["month"] == '':
            return 0.025
        else:

            for i in range(len(monthEnglish)):
                resultI = 0
                if monthEnglish[i].upper() in dd["month"]:
                    resultI = i
                    break

            return yearWeigthArray[dd["year"]][resultI]

    else:

        return 0.025


    # yearWeigthArray = {
    #     # "2009": [0.6875, 0.715, 0.7425, 0.77, 0.7975, 0.825, 0.8525, 0.88, 0.9075, 0.935, 0.9625, 1],
    #     "2016":[0.025,0.05,0.075,0.1,0.125,0.15,0.175,0.2,0.225,0.25,0.275,0.3],
    #     "2017":[0.3445,0.371,0.3975,0.424,0.4505,0.477,0.5035,0.53,0.5565,0.583,0.6095,0.636],
    #     "2018":[0.6875,0.715,0.7425,0.77,0.7975,0.825,0.8525,0.88,0.9075,0.935,0.9625,1]
    # }
    #
    # monthEnglish = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]
    #
    # newData = []
    # for dd in data:
    #
    #     tempData = {
    #         "pmid": "",
    #         "allScore": "",
    #     }
    #
    #     if dd["year"] in ",".join(yearWeigthArray.keys()):
    #         if dd["month"] == '':
    #             tempData["allScore"] = dd["allScore"] * 0.025
    #         else:
    #
    #             for i in range(len(monthEnglish)):
    #                 resultI = 0
    #                 if monthEnglish[i].upper() in dd["month"]:
    #                     resultI = i
    #                     break
    #
    #             tempData["allScore"] = dd["allScore"] * yearWeigthArray[dd["year"]][resultI]
    #
    #     else:
    #
    #         tempData["allScore"] = dd["allScore"] * 0.025
    #
    #     tempData["pmid"] = dd["pmid"]
    #     newData.append(tempData)
    #
    # return newData

        # for key,values in dd.items():
        #     if key in yearWeigthArray.keys():
        #         pass
        #     else:
        #         tempData["allScore"] = dd["allScore"] * 0.025


#计算总分
def calTotalScore(data):
    newData = []
    for dd in data:

        tempData = {
            "ut":"",
            "allScore":"",
            # "year":"",
            # "month":""
        }

        num = 0

        for key,values in dd.items():
            if key in weightArray.keys():
                try:
                    num += dd[key]
                except:
                    continue

        tempData["ut"] = dd["ut"]
        tempData["allScore"] = num
        # tempData["year"] = dd["year"]
        # tempData["month"] = dd["month"]
        newData.append(tempData)

    return newData


#乘权重
def calWeight(data):

    newData = []
    for dd in data:
        for key,values in dd.items():
            try:
                dd[key] = round(dd[key] * weightArray[key],3)
            except:
                pass
        newData.append(dd)

    return newData
#那几个字段进行归一化
def normalization(data):

    fundsFractionMax = 0
    freferenceFractionMax = 0
    j9FractionMax = 0
    z9FractionMax = 0
    downLoadsFractionMax = 0

    #计算出最大值
    for dd in data:
        if float(dd["fundsFraction"]) > float(fundsFractionMax):
            fundsFractionMax = float(dd["fundsFraction"])

        if float(dd["referenceFraction"]) > freferenceFractionMax:
            freferenceFractionMax = float(dd["referenceFraction"])

        if float(dd["j9Fraction"]) > j9FractionMax:
            j9FractionMax = float(dd["j9Fraction"])

        if float(dd["z9Fraction"]) > z9FractionMax:
            z9FractionMax = float(dd["z9Fraction"])

        if float(dd["downLoadsFraction"]) > downLoadsFractionMax:
            downLoadsFractionMax = float(dd["downLoadsFraction"])

    #归一化后的数据
    newData = []
    for dd in data:

        dd["fundsFraction"] = float(dd["fundsFraction"])
        dd["referenceFraction"] = float(dd["referenceFraction"])
        dd["j9Fraction"] = float(dd["j9Fraction"])
        dd["z9Fraction"] = float(dd["z9Fraction"])
        dd["downLoadsFraction"] = float(dd["downLoadsFraction"])



        try:
            dd["fundsFraction"] /= fundsFractionMax
        except:
            pass

        try:
            dd["referenceFraction"] /= freferenceFractionMax
        except:
            pass

        try:
            dd["j9Fraction"] /= j9FractionMax
        except:
            pass

        try:
            dd["z9Fraction"] /= z9FractionMax
        except:
            pass

        try:
            dd["downLoadsFraction"] /= downLoadsFractionMax
        except:
            pass

        newData.append(dd)

    return newData

#被引用次数  改改
def calZ9Fraction(pmid):

    sql = "select `z9` from wos_true_pm_z9 where ut='{}' limit 1".format(pmid)


    data = sqlSelect(sql)

    try:
        return int(data[0][0])
    except:

        return 0

#下载次数
def calDownFraction(pmid):

    sql = "select `Value` from AmountOfDownloads where ut='{}' limit 1".format(pmid)

    data = sqlSelect(sql)

    try:
        return int(data[0][0])
    except:

        return 0

# 替代计量指数
def calAltFraction(pmid):

    sql = "select `Value` from wos_true_pm_altmetrics where ut='{}' limit 1".format(pmid)


    data = sqlSelect(sql)

    try:
        num = int(data[0][0])

        # 归一化
        if num == 0:
            fra = 0
        if num >= 1 and num < 3:
            fra = 0.2
        if num >= 3 and num < 5:
            fra = 0.3
        if num >= 5 and num < 8:
            fra = 0.4
        if num >= 8 and num < 10:
            fra = 0.5
        if num >= 10 and num < 20:
            fra = 0.6
        if num >= 20 and num < 30:
            fra = 0.8
        if num >= 30:
            fra = 1

        return fra
    except:

        return 0


#计算中科院分区
def calZhyFraction(j9):

    sql = '''select Z_daleifenqu from zhy_nlmid_fenqu_ta where N_NLMTitleAbbreviation="{}"'''.format(j9)

    data = sqlSelect(sql)

    try:

        if int(data[0][0]) == 1:
            return 1
        if int(data[0][0]) == 2:
            return 0.75
        if int(data[0][0]) == 3:
            return 0.5
        if int(data[0][0]) == 4:
            return 0.25

    except:
        return 0
#计算影响因子
def calJ9Fraction(j9,j9WeightJson):

    sql = '''select S_JournalImpactFactor from jcr_nlmid_ta_if_02 where N_NLMTitleAbbreviation="{}"'''.format(j9)

    data = sqlSelect(sql)

    try:
        return float(j9WeightJson[j9]) * data[0][0]
    except:
        return 0

#计算语种类型分数
def calLaFraction(la):
    data = la.split("; ")
    str = ''
    for dd in data:
        ddd = '"' + dd + '"'
        str += ddd + ","

    sql = "select `Value` from classificationoflanguage where `Language` in({}) order by `Value` desc limit 1".format(
        str.strip(","))
    data = sqlSelect(sql)
    try:
        return data[0][0]
    except:
        return 0

#计算文献类型分数
def calDtFraction(dt):

    data = dt.split("; ")
    str = ''
    for dd in data:
        ddd = '"' + dd + '"'
        str += ddd + ","

    sql = "select `Value` from typeofarticle where `Type` in({}) order by `Value` desc limit 1".format(str.strip(","))
    data = sqlSelect(sql)
    try:
        return data[0][0]
    except:
        return 0

#计算页码分数
def calPageFraction(num):
    num = int(num)
    try:
        # 归一化
        if num == 0:
            return 0
        if num >= 1 and num < 2:
            return 0.1
        if num >= 2 and num < 3:
            return 0.2
        if num >= 3 and num < 5:
            return 0.3
        if num >= 5 and num < 8:
            return 0.4
        if num >= 8 and num < 10:
            return 0.5
        if num >= 10 and num < 20:
            return 0.6
        if num >= 20 and num < 30:
            return 0.8
        if num >= 30:
            return 1

    except:

        return 0

#计算参考文献的数量和质量
def caleFerenceFraction(pmid):

    sql = "select Reference,`Value` from numberofreference where ut='{}' limit 1".format(pmid)

    data = sqlSelect(sql)

    try:
        num = len(data[0][0].split("|"))

        # 归一化
        if num == 0:
            fra = 0
        if num >= 1 and num < 20:
            fra = 0.1
        if num >= 20 and num < 50:
            fra = 0.2
        if num >= 50 and num < 80:
            fra = 0.3
        if num >= 80 and num < 100:
            fra = 0.4
        if num >= 100 and num < 120:
            fra = 0.5
        if num >= 120 and num < 160:
            fra = 0.6
        if num >= 160 and num < 200:
            fra = 0.8
        if num >= 200:
            fra = 1

        return fra,data[0][1]
    except:

        return 0,0

#计算通讯作者分数
def calAuthorCommuFraction(pmid):

    sql = "select CommunicationAuthor from numberofcommunicationauthor where ut='{}' limit 1".format(pmid)

    data = sqlSelect(sql)
    try:
        sql = '''select Value from author_unit_weight where AuthorsUnit="{}" limit 1'''.format(data[0][0])
        data = sqlSelect(sql)
        return data[0][0]
    except:
        return 0

#计算作者单位得分
def calAuthorAddrNum(pmid):
    sql = "select AuthorsUnit from numberofauthors where ut='{}' limit 1".format(pmid)

    data = sqlSelect(sql)

    try:
        sql = '''select Value from author_unit_weight where AuthorsUnit="{}" limit 1'''.format(data[0][0])

        data = sqlSelect(sql)
        return data[0][0]
    except:
        return 0


#计算基金质量权重得分
def calFunds(funds):
    if funds == '':
        return 0
    else:
        data = spintFund(funds)

        #拼接字符成a,b,c这种格式方便sql用in语句查询
        str = ''
        for dd in data:
            ddd = '"' + dd + '"'
            str += ddd + ","

        fundsWeight = getFundWeightData((str.strip(",")))

        #累加
        num = 0
        for weight in fundsWeight:
            try:
                num += weight[0]
            except:
                num += 0
        return num

#计算基金数量得分
def calFundsNum( funds):
    if funds == '':
        return 0
    else:
        data = spintFund(funds)

        num  = len(data)

        # 归一化
        if num == 0:
            return 0
        if num >= 1 and num < 2:
            return 0.3
        if num >= 2 and num < 3:
            return 0.4
        if num >= 3 and num < 5:
            return 0.5
        if num >= 5 and num < 8:
            return 0.6
        if num >= 8 and num < 10:
            return 0.8
        if num >= 10:
            return 1

            # 计算基金数量得分

def calAuthorNum( author):

    if author == '':
        return 0.5
    else:
        temp = author.split(';-;')

        num = len(temp)
        # 归一化
        if num == 0:
            return 0.5
        if num >= 1 and num < 3:
            return 0.2
        if num >= 3 and num < 5:
            return 0.3
        if num >= 5 and num < 8:
            return 0.4
        if num >= 8 and num < 10:
            return 0.5
        if num >= 10 and num < 20:
            return 0.6
        if num >= 30 and num < 30:
            return 0.8
        if num >= 30:
            return 1

def isHaveKeyByJ9(request):
    return HttpResponse(json.dumps({
        "isHave":0

    }
    ), content_type="application/json")

# index(1)
# if __name__ == '__main__':
#
#     start = time.time()
#     index()
#     end = time.time()
#
#     print(end - start)


# if __name__ == '__main__':
#     fraction = fraction()
#
#     fraction.index()
